from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
# import gc
import copy
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import re
import numpy as np
from os import walk
# import matplotlib.pyplot as plt
from subprocess import call
import os

# For format1:
# step1
# Updated by Wei on 20141030 night
# output the binary file for external sort, good operation
print "Begins..."
# input arguments:
# read from the /san/share
# write to the /home
# arg1: 20000 # total number of queries needed to process
# arg2: ifn # the input file we are going to parse
# arg3: ofn # the ofn file we are going to output
# arg4: ofn2 # the log file for processing this file

if len(sys.argv) != 5:
    print "USAGE: programName.py NUM_QUERY_TO_PARSE ifn ofn ofn_log"
    exit(1)
else:
    NUM_QUERY_TO_PARSE = int(sys.argv[1])
    ifn = sys.argv[2]
    ofn = sys.argv[3]
    ofn2 = sys.argv[4]
    pathElements = sys.argv[3].strip().split("/")
    # new setting
    ofn3 = pathElements[0] + "/" + pathElements[1] + "/" + pathElements[2] + "/" + "binaryFiles" + "/" + pathElements[4] + ".binary"
    # original setting
    # ofn3 = sys.argv[3] + ".binary"

# key: term
# value: termID
termAndTermIDDict = {}
#termAndTermIDDict["tax"] = "100"
#termAndTermIDDict["deduction"] = "200"
#termAndTermIDDict["petition"] = "300"
#termAndTermIDDict["ratings"] = "400"
#termAndTermIDDict["and"] = "500"
#termAndTermIDDict["gemstones"] = "600"
#termAndTermIDDict["biography"] = "700"

ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/data/gov2_wholeLexiconTermID_Term_ListLength_sortedByAlphabeticalOrder"
ifh = open(ifn0,"r")
l = ifh.readline()
counter = 0
while l:
    le = l.strip().split(" ")
    termID = le[0]
    term = le[1]
    termAndTermIDDict[term] = termID
    l = ifh.readline()
    counter += 1
    if counter % 1000000 == 0:
        print "counter:",counter,"processed."
        # break
ifh.close()

print "Overall:"
#print "termAndTermIDDict['tax']: ",termAndTermIDDict['tax']
#print "termAndTermIDDict['deduction']: ",termAndTermIDDict['deduction']
#print "termAndTermIDDict['petition']: ",termAndTermIDDict['petition']
print "len(termAndTermIDDict):",len(termAndTermIDDict)

queryDict = {}
termPositionAndTermIDDict = {}
queryCounter = 0
# key: docID
# value: a list of tuples like the following format (termID,probability)
postingHitDict = {}
# documentHitDict = {}

ifh = open(ifn,"r")
ofh = open(ofn,"w")
ofh2 = open(ofn2,"w")
ofh3 = open(ofn3,"w")

l = ifh.readline()
while l:
    le = l.strip().split(" ")
    #print len(le)
    #print le
    if l.strip().startswith("Search"):
        currentSearchContent = l.strip()[8:]
        if currentSearchContent.strip() != "":
            queryCounter += 1
            if queryCounter % 1000 == 0:
                # print queryCounter,"queries processed."
                ofh2.write(str(queryCounter) + " " + "queries processed." + "\n")
            
            if queryCounter == NUM_QUERY_TO_PARSE:
                for i in range(0,25205179):
                    if str(i) in postingHitDict:
                        for termID in postingHitDict[str(i)]:
                            # NOT going to write anything to the disk
                            ofh.write(str(i) + " " + termID + " " + str(postingHitDict[str(i)][termID]) + "\n")
                            # write the binary format to the disk
                            ofh3.write(pack("3I",i,int(termID),postingHitDict[str(i)][termID] ))
                    else:
                        pass
                break

    if len(le) == 20 and le[1] == "Overlapping":
        currentTermPosition = int(le[0])
        currentTerm = le[4][1:-2]
        termPositionAndTermIDDict[currentTermPosition] = termAndTermIDDict[currentTerm] 
        currentTermID = termAndTermIDDict[currentTerm]
    
    # I ONLY want the TOP10 postings filled into the index
    # I only use the top10 posting rank / OR I can use up to 1000
    if len(le) == 15 and le[-2].startswith("GX") and int(le[0]) <= 1000:
        # print termPositionAndTermIDDict
        docID = le[-3]
        base = 1
        for i in range(0,len(termPositionAndTermIDDict)):
            if float(le[base + i]) != 0.0:
                if docID not in postingHitDict:
                    postingHitDict[docID] = {}
                if termPositionAndTermIDDict[i] not in postingHitDict[docID]:
                    postingHitDict[docID][termPositionAndTermIDDict[i]] = 0
                postingHitDict[docID][termPositionAndTermIDDict[i]] += 1
    
    l = ifh.readline()
ifh.close()
ofh.close()
ofh2.close()
ofh3.close()

print "Overall:"
print "ifn:",ifn
print "ofn:",ofn
print "ofn2:",ofn2
print "ofn3:",ofn3
print "Ends."



